import requests
import urllib.request as req
from urllib import error
import json
import os
import pandas as pd
import math
import time
from random import randint
import csv
from repo_crawler import *
from bs4 import BeautifulSoup


per_page = 100


#获取指定开发者信息
#计划从仓库的contributor中获取到用户的name(这一过程必须用beautiful soup实现，不然名字对不上)，再用search查询用户信息并保存在本地（其中邮箱必须用contributor的api获取）
# https://gitee.com/api/v5/users/xieXiuQ   是链接
'''
    dict中的信息:
    html_url:  URL主页
    email : email
    company : 可以用来表示address
    'followers': 99, 'following': 18, 'stared': 3, 'watched': 202
    贡献（Stars, Forks, Watches, Followers, Following）
'''
#根据仓库信息爬取contributor的用户名(输入为单个名称)***
def get_user_name(pkg_name):
    count = 1
    # 必须登录才能爬取，所以要填写用户cookie
    cookie = 'oschina_new_user=false; user_locale=zh-CN; remote_way=http; Serve_State=true; yp_riddler_id=1ce6323c-fe64-4c65-9c87-b7003dd467d4; slide_id=9; sensorsdata2015jssdkchannel={"prop":{"_sa_channel_landing_url":""}}; tz=Asia/Shanghai; Hm_lvt_24f17767262929947cc3631f99bfd274=1676857349,1677117299,1677240622,1677305657; gitee_user=true; Hm_lvt_6bc840df1e0b2cbbd5d0aab3e06b2610=1676812524,1676891467,1677240622,1677305848; sensorsdata2015jssdkcross={"distinct_id":"8989432","first_id":"186873d743d7f-0508e6ed3f6bab-74525476-1327104-186873d743ed33","props":{"$latest_traffic_source_type":"直接流量","$latest_search_keyword":"未取到值_直接打开","$latest_referrer":"","$latest_utm_source":"pzpc","$latest_utm_medium":"sem","$latest_utm_campaign":"home","$latest_utm_content":"competition","$latest_utm_term":"github"},"$device_id":"178de8f7582332-0d29e0f040ed83-7d667961-1327104-178de8f7583606","identities":"eyIkaWRlbnRpdHlfY29va2llX2lkIjoiMTg0NjZiOTNmODVmMTgtMDkxMjExNDcwMTc5OTMtN2Q1ZDU0NzQtMTMyNzEwNC0xODQ2NmI5M2Y4NmFhZSIsIiRpZGVudGl0eV9sb2dpbl9pZCI6Ijg5ODk0MzIifQ==","history_login_id":{"name":"$identity_login_id","value":"8989432"}}; Hm_lpvt_6bc840df1e0b2cbbd5d0aab3e06b2610=1677308274; Hm_lpvt_24f17767262929947cc3631f99bfd274=1677308921; gitee-session-n=WmtWK3lDMHFqSmFLNzZGR3FGQ2d0QzZBcTAxR0lGdVZlai9FandDbG1XUlVzZmxnN3FZRVRTT3c3Ymk3OGFOQjlVaFVHUExjNkQwL3ovUGFqbm13ZFp1Z1duRDg0cHpNQmpEQTAwTnJJVlcyL2I4Rm9wS2xhZ01KQ2NBYVE1UFhFZkpIOE4rNm1vKzNDcCtkRDc2OFZ4Y2dZNHg5VERuOXJ6S0U5eXZ5WEdNVDBvUGYrRTlqbVV2MzlFWmtqUDN2VWtETVN6VTAvVmxmR2k5cm9wQ3NsOWlCRTBOOEM3SGV5ZzU5Mm9tRzFCaWVKa1Exa0VSNSsyemdaNlM4UXhxZmRQUzI4aVdvRmYvZ2J3YlpkTkhRSlBTdmR5S0pUb3VwbE5XNHR5djZJNTNhVEtpZlo4ZldhZWp1MkxTaURZOGpjVXRES0FrQnYxeHI0Y3FXNnNCUEFMOWpyV3JZNGQzeDh6WG9DdUkxdU5DZlRPZnBZMis3T21URmZXSm9hQ0NxZ1RiazUyU2VCc0ozS2VQdG9NTWUzMHR0amE2ckwzT2cwRWx1Z0tHTnhUZkZGdURwU1hwSFdxc3ZBL0tWWDdYcUVhd214d3g3dG9HTml1MmI1cm5KM3dMenlWNThMUVB5YzVlWk93L2h3dmRqdThhWnFyVVkyOUV2SXNkTnBPanI5aTVWNk9PQlFWRDlLSEVzRy9Vbk5JdDVrYXArdVJxMER3UkhGOHJPMUNmZlorWHJlYVJuTHVBbFoyczlZNlZ3SkxNV3JYeVcyTEdZSCtxU1k4QXBvb0hWZzh3TmkzOE9Qc1BJdkFMR2haOXlpN1lEWFhKaS9jZWIwckJQZ25oUy0teU5DQ3BhUkNkUVM5UWxRaFdJTERadz09--8fad856e3e99a17527b044340dade3fffa9dcae3'
    cookie = cookie.encode('utf-8')
    random_agent = USER_AGENTS[randint(0, len(USER_AGENTS) - 1)]
    headers = {
        'User-Agent':  random_agent,
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'Cookie': cookie
    }
    contributors = []
    url = "https://gitee.com/src-openeuler/"+pkg_name+"/contributors?ref=master"
    response = requests.get(url, headers=headers)
    text = response.text
    soup = BeautifulSoup(text, 'lxml')
    package_name = soup.findAll(name='div', class_='item user-list-item')
    for i in package_name:
        contributor = i.a.attrs.get('href')    #爬取到用户的name，其中包含了/和邮箱两种类型
        if "@" not in contributor:
            contributor = contributor[1:]
        contributors.append(contributor)

    return contributors

#获取用户邮箱(输入为单个名称)***
def get_user_email(pkg_name):
    email = []
    api_key = API_KEY[randint(0, len(API_KEY) - 1)]
    random_agent = USER_AGENTS[randint(0, len(USER_AGENTS) - 1)]
    headers = {'User-Agent': random_agent}
    url = 'https://gitee.com/api/v5/repos/src-openEuler/'+pkg_name+'/contributors?type=authors&api_key=%s&access_token=639da62c28e90262fbbfe3a9cd9b68d3' % (api_key)
    response = requests.get(url, headers=headers)
    response_dict = response.json()
    for item in response_dict:
        email.append(item['email'])
    time.sleep(0.3)
    return email

#将email信息写入csv文件
def write_email_to_csv(pkg_name):
    count = 1
    with open('emails.csv', mode='a', encoding='UTF8', newline='') as f:
        writer = csv.writer(f)
        # write the data
        for item in pkg_name:
            email = get_user_email(item)
            writer.writerow(email)
            if count % 5 == 0:
                print("already save %s nodes" % count)
            count += 1
    return

def write_name_to_csv(pkg_name):
    count = 1
    with open('names.csv', mode='a', encoding='UTF8', newline='') as f:
        writer = csv.writer(f)
        # write the data
        for item in pkg_name:
            names = get_user_name(item)
            writer.writerow(names)
            if count % 10 == 0:
                print("already save %s nodes" % count)
            count += 1
    return
#根据用户名获取用户信息
def get_user_info(names,pkg,emails):
    # 构建API请求
    users = []
    api_key = API_KEY[randint(0, len(API_KEY) - 1)]
    random_agent = USER_AGENTS[randint(0, len(USER_AGENTS) - 1)]
    headers = {'User-Agent': random_agent}
    if len(names)==len(emails):
        flag = 1
    else:
        flag = 0
    for i in range(len(names)):
        user_name = names[i]
        if "@" in user_name:
            continue
        url = "https://gitee.com/api/v5/users/" + user_name + "?&api_key=%s&access_token=cd97ab05bbf438384e9097dc0bb2da11" % (api_key)
        response = requests.get(url, headers=headers)
        if response.status_code==404:   #如果报错则继续
            continue
        response_dict = response.json()
        if flag == 1:
            response_dict["email"] = emails[i]
        users.append(response_dict)
        print("successfully get %s user_info !\n" % user_name)
        time.sleep(0.1)
    filename = 'users/'+ pkg +'_users'+'.json'
    with open(filename, 'w') as file_obj1:
        json.dump(users, file_obj1)
    return users

#获取已经存在本地的用户name和email
def get_csv():
    filename1 = 'names.csv'
    filename2 = 'emails.csv'
    names = []
    emails = []
    with open(filename1, encoding='gbk', newline='') as csvfile1:
        csv_reader = csv.reader(csvfile1)  # 使用csv.reader读取csvfile中的文件
        # header = next(csv_reader)        # 读取第一行每一列的标题
        for row in csv_reader:  # 将csv 文件中的数据保存到data中
            while '' in row:
                row.remove('')
            names.append(row)  # 选择某一列加入到data数组中

    with open(filename2, encoding='gbk', newline='') as csvfile2:
        csv_reader = csv.reader(csvfile2)  # 使用csv.reader读取csvfile中的文件
        # header = next(csv_reader)        # 读取第一行每一列的标题
        for row in csv_reader:  # 将csv 文件中的数据保存到data中
            while '' in row:
                row.remove('')
            emails.append(row)  # 选择某一列加入到data数组中


    return names,emails


if __name__ == '__main__':
    pkg_name = get_pkg_name()
    #获取emails
    # write_email_to_csv(pkg_name)
    #获取names
    # write_name_to_csv(pkg_name[8000:])

    #获取user info
    names,emails = get_csv()
    # print(pkg_name[432])
    # exit(1)
    for i in range(len(pkg_name)):
        get_user_info(names[i],pkg_name[i],emails[i])





